{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random starting synaptic weights: \n",
      "[[-0.16595599]\n",
      " [ 0.44064899]\n",
      " [-0.99977125]]\n",
      "error after 0 iterations: 0.578374046722\n",
      "error after 1000 iterations: 0.0353771814512\n",
      "error after 2000 iterations: 0.024323319584\n",
      "error after 3000 iterations: 0.0196075022358\n",
      "error after 4000 iterations: 0.016850233908\n",
      "error after 5000 iterations: 0.014991814044\n",
      "error after 6000 iterations: 0.0136320935305\n",
      "error after 7000 iterations: 0.01258242301\n",
      "error after 8000 iterations: 0.0117408289409\n",
      "error after 9000 iterations: 0.0110467781322\n",
      "New synaptic weights after training: \n",
      "[[ 12.79547496]\n",
      " [ -4.2162058 ]\n",
      " [ -4.21608782]]\n",
      "Considering new situation [1, 0, 0] -> ?: \n",
      "[ 0.99999723]\n"
     ]
    }
   ],
   "source": [
    "# from https://iamtrask.github.io//2015/07/12/basic-python-network/\n",
    "import numpy\n",
    "\n",
    "class NeuralNetwork():\n",
    "    def __init__(self):\n",
    "        # Seed the random number generator, so it generates the same numbers\n",
    "        # every time the program runs.\n",
    "        random.seed(1)\n",
    "\n",
    "        # We model a single neuron, with 3 input connections and 1 output connection.\n",
    "        # We assign random weights to a 3 x 1 matrix, with values in the range -1 to 1\n",
    "        # and mean 0.\n",
    "        self.synaptic_weights = 2 * random.random((3, 1)) - 1\n",
    "\n",
    "    # The Sigmoid function, which describes an S shaped curve.\n",
    "    # We pass the weighted sum of the inputs through this function to\n",
    "    # normalise them between 0 and 1.\n",
    "    def __sigmoid(self, x):\n",
    "        return 1 / (1 + exp(-x))\n",
    "\n",
    "    # The derivative of the Sigmoid function.\n",
    "    # This is the gradient of the Sigmoid curve.\n",
    "    # It indicates how confident we are about the existing weight.\n",
    "    def __sigmoid_derivative(self, x):\n",
    "        return x * (1 - x)\n",
    "\n",
    "    # We train the neural network through a process of trial and error.\n",
    "    # Adjusting the synaptic weights each time.\n",
    "    def train(self, training_set_inputs, training_set_outputs, number_of_training_iterations):\n",
    "        for iteration in iter(range(number_of_training_iterations)):\n",
    "            # Pass the training set through our neural network (a single neuron).\n",
    "            output = self.think(training_set_inputs)\n",
    "\n",
    "            # Calculate the error (The difference between the desired output\n",
    "            # and the predicted output).\n",
    "            error = training_set_outputs - output\n",
    "\n",
    "            # Multiply the error by the input and again by the gradient of the Sigmoid curve.\n",
    "            # This means less confident weights are adjusted more.\n",
    "            # This means inputs, which are zero, do not cause changes to the weights.\n",
    "            adjustment = dot(training_set_inputs.T, error * self.__sigmoid_derivative(output))\n",
    "\n",
    "            # Adjust the weights.\n",
    "            self.synaptic_weights += adjustment\n",
    "            if (iteration % 1000 == 0):\n",
    "                print (\"error after %s iterations: %s\" % (iteration, str(numpy.mean(numpy.abs(error)))))\n",
    "\n",
    "    # The neural network thinks.\n",
    "    def think(self, inputs):\n",
    "        # Pass inputs through our neural network (our single neuron).\n",
    "        return self.__sigmoid(dot(inputs, self.synaptic_weights))\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    #Intialise a single neuron neural network.\n",
    "    neural_network = NeuralNetwork()\n",
    "\n",
    "    print (\"Random starting synaptic weights: \")\n",
    "    print (neural_network.synaptic_weights)\n",
    "\n",
    "    # The training set. We have 4 examples, each consisting of 3 input values\n",
    "    # and 1 output value.\n",
    "    training_set_inputs = array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 0]])\n",
    "    training_set_outputs = array([[0, 1, 1, 0]]).T\n",
    "\n",
    "    # Train the neural network using a training set.\n",
    "    # Do it 10,000 times and make small adjustments each time.\n",
    "    neural_network.train(training_set_inputs, training_set_outputs, 10000)\n",
    "\n",
    "    print (\"New synaptic weights after training: \")\n",
    "    print (neural_network.synaptic_weights)\n",
    "\n",
    "    # Test the neural network with a new pattern\n",
    "    test = [1, 0, 0]\n",
    "    print (\"Considering new situation %s -> ?: \" % test )\n",
    "    print (neural_network.think(array(test)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input:\n",
      "[[0 0 1]\n",
      " [1 1 1]\n",
      " [1 0 1]\n",
      " [0 1 0]]\n",
      "truth:\n",
      "[[0]\n",
      " [1]\n",
      " [1]\n",
      " [0]]\n",
      "dot-product:\n",
      "[[-0.99977125]\n",
      " [-0.72507825]\n",
      " [-1.16572724]\n",
      " [ 0.44064899]]\n",
      "output:\n",
      "[[ 0.2689864 ]\n",
      " [ 0.3262757 ]\n",
      " [ 0.23762817]\n",
      " [ 0.60841366]]\n",
      "error:\n",
      "[[ 0.2689864 ]\n",
      " [-0.6737243 ]\n",
      " [-0.76237183]\n",
      " [ 0.60841366]]\n",
      "derivative:\n",
      "[[-0.28621005]\n",
      " [-0.00314557]\n",
      " [-0.23331852]]\n"
     ]
    }
   ],
   "source": [
    "# a look inside one iteration\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + exp(-x))\n",
    "\n",
    "def sigmoid_derivative(x):\n",
    "    return x * (1 - x)\n",
    "    \n",
    "sweights = [[-0.16595599],[ 0.44064899],[-0.99977125]]\n",
    "print (\"input:\")\n",
    "print (training_set_inputs)\n",
    "print (\"truth:\")\n",
    "print (training_set_outputs)\n",
    "print (\"dot-product:\")\n",
    "print (dot(training_set_inputs, sweights) )\n",
    "output = sigmoid(dot(training_set_inputs, sweights))\n",
    "print (\"output:\")\n",
    "print (output)\n",
    "error = sigmoid(dot(training_set_inputs, sweights)) - training_set_outputs\n",
    "print (\"error:\")\n",
    "print (error)\n",
    "print (\"derivative:\")\n",
    "print (dot(training_set_inputs.T, error * sigmoid_derivative(output)) )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
